import os
import scrapy


class BiqugeSpider(scrapy.Spider):
    name = 'biquge'
    # allowed_domains = ['www.xxx.com']
    start_urls = ['http://www.156n.com/wanben/']

    if not os.path.exists('./笔趣阁'):
        os.mkdir('./笔趣阁')

    def parse(self, response):
        book_list = response.xpath('//*[@id="newscontent"]/div[1]/ul/li')
        for book in book_list:
            book_title = book.xpath('./span[2]/a/text()').extract_first()
            book_type = book.xpath('./span[1]/text()').extract_first()
            book_author = book.xpath('./span[4]/text()').extract_first()
            book_url = book.xpath('./span[2]/a/@href').extract_first()
            new_book_url = 'http://www.156n.com' + book_url
            path = './笔趣阁' + '/' + book_title
            if not os.path.exists(path):
              os.mkdir(path)
            yield scrapy.Request(new_book_url,callback=self.chapt_parse,meta={'path':path})

    def chapt_parse(self,response):
        path = response.meta['path']
        chapt_list = response.xpath('//*[@id="list"]//dd')
        for chapt in chapt_list:
            chapt_title = chapt.xpath('./a/text()').extract_first()
            chapt_url = chapt.xpath('./a/@href').extract_first()
            new_book_url = 'http://www.156n.com' + chapt_url
            yield scrapy.Request(new_book_url,self.content_parse,meta={'path':path,'name':chapt_title})

    def content_parse(self,response):
        path = response.meta['path']
        chaptTitle = response.meta['name']
        content = response.xpath('//*[@id="content"]//p/text()').extract()
        content = ''.join(content)
        path = path + '/' + chaptTitle + '.text'
        with open(path,'w',encoding='utf-8') as fp:
            fp.write(content)










   
        
         